Notebook to test basic functionalities¶
Imports¶
In [1]:
# Settings for notebook visualization
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'
%matplotlib inline
from IPython.core.display import HTML
HTML("""<style>.output_png img {display: block;margin-left: auto;margin-right: auto;text-align: center;vertical-align: middle;} </style>""")
Out[1]:
In [2]:
# Necessary imports
import os
import numpy as np
import pandas as pd
import matplotlib as plt
import quantstats as qs
from datetime import datetime, timedelta
print("Libraries imported correctly")
In [3]:
os.chdir("/Users/Sergio/Documents/Master_QF/Thesis/Code/Algorithmic Strategies")
%run Functions.ipynb
Data¶
In [4]:
ini_equity_default = 100
commision_default = 2/130000 + 12.5/130000 #0.000111538462, around 0.011..% of the equity
In [5]:
%run Functions.ipynb
#data = get_sp500_data(from_local_file=False, save_to_file=True)
data = get_sp500_data(from_local_file=True, save_to_file=False)
data = data[['Open', 'Close']]
data['Market_daily_ret'] = data['Close'].pct_change().fillna((data['Close']-data['Open'])/data['Open'])
data = data.loc['2000':'2020' ,['Close', 'Market_daily_ret']]
data.tail()
data['Close'].plot(title='SP500', legend=True)
Out[5]:
Out[5]:
Backtest of two years, using 2 different combinations of MA crossover¶
This is useful to see the result of doing two backtests with different parameters on two different periods. Finally those periods are merged together.
This is a backtest from 2018-01-01 to 2019-12-31. Each year will have a different combination of MAs.
- 2018: will be long all the time.
- 2019: will use fast_ma=75 and slow_ma=200.
1st year finishes with a long position, and 2nd year starts with a neutral position. This will be useful to see again that 'Strat_position' and 'Costs' are calculated and added correctly to the overall performance.
In [6]:
date_fmt = '%Y-%m-%d'
first_day = datetime.strptime('2018-01-01', date_fmt)
last_day = datetime.strptime('2019-12-31', date_fmt)
fast_ma = 75
slow_ma = 200
Performance of each year individually:¶
The purpose of each plots is:
- See performance of Market and Long only strategy during 2018. 'Strat_position' and 'Long_only' are always 1.
- See performance of Market and MA (75-200) strategy during 2019. See how 'Strat_position' comes from 0 and changes to 1 on the first day.
In [7]:
%run Functions.ipynb
df = data[first_day:last_day].copy()
commision = commision_default
cols = ['Strat_daily_ret', 'Strat_position', 'Costs', 'Long_only', 'Market_cum_ret', 'Strat_cum_ret']
tmp_df = pd.DataFrame(columns=cols)
df_year_1 = df.loc[str(first_day.year)].copy()
df_year_2 = df.loc[str(last_day.year)].copy()
strategy_1 = buy_and_hold(df_year_1)
strategy_2 = ma_crossover(df_year_2, fast_ma, slow_ma)
# Year 1
df_year_1 = backtest_print_plot(df_year_1, strategy_1, previous_position=0, commision=commision, with_legend=True)
last_position = df_year_1.loc[df_year_1.index[-2], 'Strat_position'] # Position during the last day
last_equity = df_year_1.loc[df_year_1.index[-1], 'Market_cum_ret']
# last_equity = 100
# Year 2
df_year_2 = backtest_print_plot(df_year_2, strategy_2, strat_params=(fast_ma, slow_ma), previous_position=last_position, ini_equity=last_equity, commision=commision, with_legend=True)
# Put together both periods
tmp_df = pd.concat([df_year_1, df_year_2], axis=0)
# Add all obtained columns in the original df
df = pd.concat([df, tmp_df[cols]], axis=1)
# To see details of the output
df[str(df_year_1.index[0].year)].head(1)
df[str(df_year_1.index[0].year)].tail(1)
#last_equity
df[str(df_year_2.index[0].year)].head(1)
df[str(df_year_2.index[0].year)].tail(1)
# df_year_1[['Strat_position', 'Long_only']].plot()
# df_year_1[['Costs']].plot()
# df_year_2[['Costs']].plot()
Out[7]:
Out[7]:
Out[7]:
Out[7]:
Performance of both time periods together¶
The purpose of each plots is:
- Show previous plots concatenated. See that 'Strat_position' and 'Long_only' change properly in the beggining of 2019
- Subplots to see how the changes in the strategy position affect 'Long_only', 'Strat_position', and 'Costs'
In [8]:
%run Functions.ipynb
results_df = prepare_oos_df(df.loc[first_day:last_day].copy(), commision=commision_default)
show_oos_plot(results_df, with_legend=True)
print("Period: {:%Y-%m-%d} to {:%Y-%m-%d}".format(results_df.index[0], results_df.index[-1]))
print("\tOverall return of SP500: {:.2f} %. SR of SP500: {:.2f}".format(results_df['Market_cum_ret'][-1], results_df['Market_daily_ret'].sharpe()))
print("\tOverall return of our strategy: {:.2f} %. Sharpe ratio strategy: {:.2f}".format(results_df['Strat_cum_ret'][-1], results_df['Strat_cum_ret'].sharpe()))
cols = ['Long_only', 'Strat_position', 'Costs']
results_df[cols].plot(subplots=True, title='Strategy Postition and costs (in USD)', color=('tab:brown', 'r', 'k'));
plt.show()
results_df['Costs'].plot(title='Transaction Costs in USD (with more zoom)', color=('k'));
Checking 'Strat_position' and 'Costs' of last days of year 1, and first days of year 2¶
The purpose of this section is to check how 'Strat_position' and 'Costs' have changed in the dataframe. More specifically, on dates:
Start of backtesting period: 'Costs' to open the position for the following day and its effect on 'Strat_cum_ret' should be shown
First days of 2019. Since we need to change the 'Strat_position' from long to being neutral, 'Costs' should be shown for closing the position. We can also see the change in 'Long_only' from 1 to 0.
- At the end of 2019-04-23, there is an upward MA cross. 'Strat_position' and 'Costs' should change accordingly.
In [9]:
def style_original_value(x):
color = 'background-color: darkorange'
df1 = pd.DataFrame('', index=x.index, columns=x.columns)
df1.loc['2018-12-31', ['Strat_daily_ret', 'Strat_position', 'Costs', 'Strat_cum_ret']] = color
df1.loc['2019-01-02', ['Strat_daily_ret', 'Costs', 'Strat_cum_ret']] = color
return df1
def style_fixed_value(x):
color = 'background-color: green'
df1 = pd.DataFrame('', index=x.index, columns=x.columns)
df1.loc['2018-12-31', ['Strat_daily_ret', 'Strat_position', 'Costs', 'Strat_cum_ret']] = color
df1.loc['2019-01-02', ['Strat_daily_ret', 'Costs', 'Strat_cum_ret']] = color
return df1
df['2018-12-28':'2019-01-02'].style.apply(style_original_value, axis=None)
results_df['2018-12-28':'2019-01-02'].style.apply(style_fixed_value, axis=None)
Out[9]:
Out[9]:
In [10]:
%run Functions.ipynb
results_df = prepare_oos_df(df.loc[first_day:last_day].copy(), commision=0.005)
last_day_2018 = results_df[str(first_day.year)].index[-1]
first_day_2019 = results_df[str(last_day.year)].index[0]
# df contains the output df from both backtests concatenated
# results_df contains the "fixed" df.
print("First two days: \n\tWe can see how we did not enter the market until the end of the first trading day.\n\t"
"So the only returns from the first day come from transaction costs, that were added when we entered a long position\n\t"
"Costs = $ 0.5, because commision was set to 0.005 in this test")
results_df.head(2)
print("End of first OOS period and beginning of second OOS period:\n\t"
"Following two dataframes allow to see how the transaction costs from the first day of second year is moved to the last day of the first year.\n\t"
"(Because we change the position right before the close of the first year)\n")
print("Original df:")
df['2018-12-28':'2019-01-03']
print("Fixed df:\n\t"
"We can see how Strat_position and Costs were fixed properly from the original df to the fixed df:\n\t"
"\tOn 2018-12-31: \n\t"
"\tCosts (in USD) = previous(Strat_cum_ret) * [1+[Market_Daily_ret*previous(Strat_position)]] * commission\n\t"
"\tCosts (in USD) = 92.508301 * (1+(0.008492*1.0)) * (0.005) = 0.46647")
results_df['2018-12-28':'2019-01-03']
print("Change of strategy position (at the end of 2019-04-24): \n\tThere is a MA crossover at the 'Close' of 2019-04-24. A change in Strat_position should be shown,"
" with a cost to be paid at the end of the day\n\t"
"\tOn 2019-04-24: \n\t"
"\tCosts (in USD) = previous(Strat_cum_ret) * [1+[Market_Daily_ret*previous(Strat_position)]] * commission\n\t"
"\tCosts (in USD) = 92.827457 * (1+(-0.002192*0.0)) * (0.005) = 0.464137")
results_df.loc['2019-04-22':'2019-04-25']
print("Last two days. We can see information about cummulative returns from the benchmark and out strategy. \n\t"
"We have NaN in 'Strat_position', because we have not decided a strategy position for the following OOS period\n")
#df.tail(2)
results_df.tail(2)
show_oos_plot(results_df, with_legend=True)
Out[10]:
Out[10]:
Out[10]:
Out[10]:
Out[10]:
Walk forward optimization.¶
In [11]:
%run Functions.ipynb
# fast_ma_list = [1, 3, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65]
# slow_ma_list = [5, 10, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240, 260]
fast_ma_list = [5, 10, 15, 20, 25, 30, 500, 500]
slow_ma_list = [50, 100, 150, 175, 200, 225, 250, 300]
IS_start_years = pd.date_range(start='2015-01-01', end='2016-01-01', freq='1YS', closed=None)
IS_end_years = pd.date_range(start='2017-12-31', end='2018-12-31', freq='1Y', closed=None)
OOS_start_years = pd.date_range(start='2018-01-01', end='2019-01-01', freq='1YS', closed=None)
OOS_end_years = pd.date_range(start='2018-12-31', end='2019-12-31', freq='1Y', closed=None)
# IS_start_years = pd.date_range(start='1997-01-01', end='2017-01-01', freq='1YS', closed=None)
# IS_end_years = pd.date_range(start='1999-12-31', end='2019-12-31', freq='1Y', closed=None)
# OOS_start_years = pd.date_range(start='2000-01-01', end='2020-01-01', freq='1YS', closed=None)
# OOS_end_years = pd.date_range(start='2000-12-31', end='2020-12-31', freq='1Y', closed=None)
print_periods(IS_start_years, IS_end_years, OOS_start_years, OOS_end_years)
num_neighbors_matrix = get_num_neighbors(fast_ma_list, slow_ma_list)
In [12]:
%run Functions.ipynb
df_walk_forward = data[IS_start_years[0]:OOS_end_years[-1]].copy()
df_walk_forward = df_walk_forward.loc[:'2020-05-02']
# Copy data and create necessary columns
new_cols = ['Strat_daily_ret', 'Strat_position', 'Long_only', 'Costs', 'Market_cum_ret']
df_walk_forward = df_walk_forward.reindex(columns = df_walk_forward.columns.tolist() + new_cols)
last_position = 0 # We suppose that we start not being invested. 1:long, -1:short
optimization_results_list = []
robust_optimization_results_list = []
market_ir_list = []
num_loop=0
for IS_start, IS_end, OOS_start, OOS_end in zip(IS_start_years, IS_end_years, OOS_start_years, OOS_end_years):
#if num_loop == 1: break
num_loop += 1
strats_ir_matrix = np.zeros((len(fast_ma_list),len(slow_ma_list)))
strat_pnl_matrix = np.zeros((len(fast_ma_list),len(slow_ma_list)))
is_period = df_walk_forward.loc[IS_start:IS_end].copy()
oos_period = df_walk_forward.loc[OOS_start:OOS_end].copy()
############################################ IN SAMPLE ############################################
strat_pnl_matrix, strats_ir_matrix, market_pnl, market_ir = run_all_combinations(is_period, fast_ma_list, slow_ma_list, last_position)
optimization_results_list.append(strats_ir_matrix)
market_ir_list.append(market_ir)
# Get index of maximum IR for the In-Sample period, or buy_and_hold if it performed better
fast_index, slow_index, robust_strats_ir_matrix = get_best_combination(strats_ir_matrix, market_ir, num_neighbors_matrix, allow_long_only=False)
robust_optimization_results_list.append(robust_strats_ir_matrix)
if robust_strats_ir_matrix[fast_index, slow_index] > market_ir:
fast_ma_best = fast_ma_list[fast_index]
slow_ma_best = slow_ma_list[slow_index]
else: # If buy_and_hold performed better, we select buy_and_hold for the OOS
fast_ma_best = 0
slow_ma_best = 0
print("Best In-sample performance: ")
print_backtest_stats(is_period, strat_pnl_matrix[fast_index, slow_index], strats_ir_matrix[fast_index, slow_index],
market_pnl, market_ir,
strat_params=(fast_ma_best, slow_ma_best))
show_both_heatmaps(strats_ir_matrix, robust_strats_ir_matrix, market_ir, plot_title=str(IS_start.year), x_title="Slow MA", x_values=slow_ma_list, y_title="Fast MA", y_values=fast_ma_list)
############################################ OUT OF SAMPLE ############################################
print("OOS performance:")
strategy = ma_crossover(oos_period, fast_ma_best, slow_ma_best)
oos_period, last_position, strat_pnl, strat_ir, market_pnl, market_ir = backtest_strat(oos_period, strategy, previous_position=last_position)
print_backtest_stats(oos_period, strat_pnl, strat_ir, market_pnl, market_ir,
strat_params=(fast_ma_best, slow_ma_best))
# Save come columns from OOS
df_walk_forward.loc[OOS_start:OOS_end, new_cols] = oos_period[new_cols]
print()
print("End of Walk Forward Optimization")
In [13]:
%run Functions.ipynb
ini_mon = df_walk_forward[str(OOS_start_years[0].year-1)].iloc[-1]['Close']
#ini_mon = 100
results_df = prepare_oos_df(df_walk_forward.loc[OOS_start_years[0]:OOS_end_years[-1]].copy(), ini_equity=ini_mon)
results_df.iloc[[0,-2,-1]]
show_oos_plot(results_df, with_legend=True, with_signals=True)
#plt.show()
#results_df['Costs'].cumsum().plot();
Out[13]:
In [14]:
%run Functions.ipynb
from ipywidgets import widgets
options = [year for year in IS_start_years.year]
selection_slider = widgets.SelectionSlider(
options=options,
description='Starting IS year',
orientation='horizontal',
layout={'width': '900px'},
)
#selection_slider
def plot_heatmap(year):
index = year - IS_start_years.year[0]
show_both_heatmaps(optimization_results_list[index], robust_optimization_results_list[index], market_ir_list[index], plot_title=str(year), x_title="Slow MA", x_values=slow_ma_list, y_title="Fast MA", y_values=fast_ma_list)
widgets.interact(
plot_heatmap,
year=selection_slider
);
Calculate performance metrics¶
In [15]:
%run Functions.ipynb
print("Initial invested capital = {:.2f}".format(ini_mon))
results_df[['Strat_cum_ret', 'Market_cum_ret']].iloc[[0,1,-2,-1]]
metrics = calculate_performance_metrics(results_df, strat_name='MA Crossover')
metrics
show_oos_plot(results_df, with_legend=True, with_signals=True)
Out[15]:
Out[15]:
Save results¶
After the walk-forward optimization we save the results. The most important column is 'Strat_position'. From it, we can obtain daily and cummulative returns of the strategy.
In [16]:
filename = "Test_results_ma.csv"
# results_df.to_csv("data/{}".format(filename))
if filename not in os.listdir("data"):
results_df.to_csv("data/{}".format(filename))
print("File saved properly")
else:
print("File already exists")